{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "公众号 = \"温酒居\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "fn = { \"output\" : { \"公众号_htm_snippets\": \"公众号_htm_snippets_{公众号}.tsv\",\n",
    "                    \"公众号_df\": \"公众号_df_{公众号}.tsv\",\n",
    "                    \"公众号_xlsx\": \"公众号_url_{公众号}.xlsx\" } \\\n",
    "      }"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "from lxml.html import fromstring\n",
    "import time\n",
    "from random import random\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 使用Selenium\n",
    "* 要更改 opts.binary_location 至自己本地的Chrome浏览器，建议portable\n",
    "* Chrome浏览器 和 chromedriver.exe要同版本号到小数后一位\n",
    "* 要确保可以 开启浏览器机器人\n",
    "* 要确保浏览器机器人 可以打开网页 driver.get(\"https://mp.weixin.qq.com\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "import selenium"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "# coding=utf-8\n",
    "from selenium import webdriver\n",
    "import time\n",
    "\n",
    "wd = webdriver.Chrome()\n",
    "wd.get(\"https://www.baidu.com\")    # 打开百度浏览器\n",
    "wd.find_element_by_id(\"kw\").send_keys(\"selenium\")   # 定位输入框并输入关键字\n",
    "wd.find_element_by_id(\"su\").click()   #点击[百度一下]搜索\n",
    "time.sleep(3)   #等待3秒\n",
    "wd.quit()   #关闭浏览器"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "<ipython-input-7-783dcc5c7ea8>:19: DeprecationWarning: use options instead of chrome_options\n",
      "  driver = webdriver.Chrome( chrome_options = opts) #desired_capabilities=caps,\n"
     ]
    }
   ],
   "source": [
    "from selenium import webdriver\n",
    "from selenium.webdriver.common.desired_capabilities import DesiredCapabilities\n",
    "\n",
    "\n",
    "#caps=dict()\n",
    "#caps[\"pageLoadStrategy\"] = \"none\"   # Do not wait for full page load\n",
    "\n",
    "opts = webdriver.ChromeOptions()\n",
    "opts.add_argument('--no-sandbox')#解决DevToolsActivePort文件不存在的报错\n",
    "opts.add_argument('window-size=1920x3000') #指定浏览器分辨率\n",
    "opts.add_argument('--disable-gpu') #谷歌文档提到需要加上一这个属性来规避bug\n",
    "opts.add_argument('--hide-scrollbars') #隐藏滚动条, 应对些特殊页面\n",
    "#opts.add_argument('blink-settings=imagesEnabled=false') #不加载图片, 提升速度\n",
    "#opts.add_argument('--headless') #浏览器不提供可视化页面. linux下如果系统不支持可视化不加这条会启动失败\n",
    "# opts.binary_location = \"C:\\portable\\PortableApps\\IronPortable\\App\\Iron\\chrome.exe\"\n",
    "# opts.binary_location = \"C:\\Program Files (x86)\\Google\\Chrome\\Application\\chromedriver.exe\" #\"H:\\_coding_\\Gitee\\InternetNewMedia\\CapstonePrj2016\\chromedriver.exe\"  \n",
    "\n",
    "# \"H:\\_coding_\\Gitee\\InternetNewMedia\\CapstonePrj2016\\chromedriver.exe\"  \n",
    "driver = webdriver.Chrome( chrome_options = opts) #desired_capabilities=caps,"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "driver.get(\"https://mp.weixin.qq.com\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 填表登入"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "payload =  {\"account\": \"1429748618@qq.com\", \"password\": \"LOVE7788\"}\n",
    "# payload =  {\"account\": \"NFUHacks@163.com\", \"password\": \"NFU706947580\"}\n",
    "driver.find_element_by_xpath('//div[@class=\"login__type__container login__type__container__scan\"]/a').click()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "driver.find_element_by_xpath('//form[@class=\"login_form\"]//input[@name=\"account\"]').clear()\n",
    "driver.find_element_by_xpath('//form[@class=\"login_form\"]//input[@name=\"account\"]').send_keys(payload['account'])\n",
    "driver.find_element_by_xpath('//form[@class=\"login_form\"]//input[@name=\"password\"]').clear()\n",
    "driver.find_element_by_xpath('//form[@class=\"login_form\"]//input[@name=\"password\"]').send_keys(payload['password'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "driver.find_element_by_xpath('//div[@class=\"login_btn_panel\"]/a').click()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 点选单"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'展开'"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "element = driver.find_element_by_xpath('//a[@class=\"weui-desktop-btn__head-opr weui-desktop__unfold-menu-opr\"]')\n",
    "element.click()\n",
    "main_content = element.get_attribute('innerHTML')\n",
    "main_content"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "#滑动展开的菜单栏，寻找新建图文信息的选项\n",
    "driver.execute_script(\"window.scrollTo(0,document.body.scrollHeight)\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 新建图文消息"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "#利用xpath点击素材管理项\n",
    "element = driver.find_element_by_xpath('//*[@id=\"js_mp_sidemenu\"]/div/div/ul/li[2]/ul/li[1]/ul/li[1]/a')\n",
    "element.click()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "#点击新建图文信息空白处，响应折叠菜单\n",
    "element = driver.find_element_by_xpath('//*[@id=\"js_main\"]/div[3]/div[2]/div/div/div/div[1]/div/div[1]/div[1]/i')\n",
    "element.click()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "#打开折叠菜单点击第一项新建\n",
    "element = driver.find_element_by_xpath('//*[@id=\"js_main\"]/div[3]/div[2]/div/div/div/div[1]/div/div[1]/div[2]/ul/li[1]/a')\n",
    "element.click()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 新建图文消息开了另一分视窗，所以要切换 switch_to \n",
    "driver.switch_to.window(driver.window_handles[-1])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 超链接"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "超链接\n"
     ]
    }
   ],
   "source": [
    "element = driver.find_element_by_xpath('//*[text()[contains(.,\"超链接\")]]') \n",
    "main_content = element.get_attribute('innerHTML')\n",
    "print(main_content)\n",
    "element.click()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "选择其他公众号\n"
     ]
    }
   ],
   "source": [
    "# 点 选择其他公众号\n",
    "element = driver.find_element_by_xpath('//*[text()[contains(.,\"选择其他公众号\")]]') \n",
    "main_content = element.get_attribute('innerHTML')\n",
    "print(main_content)\n",
    "element.click()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "driver.find_element_by_xpath('//form//div[@class=\"inner_link_account_area\"]//input[@class=\"weui-desktop-form__input\"]').clear()\n",
    "driver.find_element_by_xpath('//form//div[@class=\"inner_link_account_area\"]//input[@class=\"weui-desktop-form__input\"]').send_keys(公众号)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<div class=\"weui-desktop-icon weui-desktop-icon__search weui-desktop-icon__small\" style=\"width: 20px; height: 20px;\"><!----> <!----> <!----> <!----> <!----> <!----> <!----> <!----> <!----> <!----> <!----> <!----> <!----> <!----> <!----> <!----> <!----> <!----> <!----> <!----> <!----> <!----> <!----> <!----> <!----> <!----> <!----> <!----> <!----> <!----> <!----> <!----> <!----> <!----> <!----> <!----> <!----> <!----> <!----> <!----> <!----> <!----> <!----> <!----> <!----> <!----> <!----> <!----> <!----> <!----> <!----> <!----> <!----> <!----> <!----> <!----> <!----> <!----> <!----> <!----> <!----> <!----> <!----> <!----> <!----> <!----> <!----> <!----> <!----> <!----> <!----> <!----> <!----> <!----> <!----> <!----> <!---->     <svg viewBox=\"0 0 24 24\" version=\"1.1\" xmlns=\"http://www.w3.org/2000/svg\" xmlns:xlink=\"http://www.w3.org/1999/xlink\"><title>MP/Icon/Search</title> <g id=\"MP/Icon/Search\" stroke=\"none\" stroke-width=\"1\" fill=\"none\" fill-rule=\"evenodd\"><path d=\"M5.78025253,5.78248558 C8.51392257,3.04881554 12.9460774,3.04881554 15.6797475,5.78248558 C18.1730922,8.27583028 18.3922898,12.1821488 16.3373403,14.9239313 L20.6294949,19.2175144 L19.2152814,20.631728 L14.922508,16.3389663 C12.180685,18.394566 8.27384272,18.1755707 5.78025253,15.6819805 C3.04658249,12.9483105 3.04658249,8.51615562 5.78025253,5.78248558 Z M6.8409127,6.84314575 C4.6930291,8.99102935 4.6930291,12.4734367 6.8409127,14.6213203 C8.98879631,16.7692039 12.4712037,16.7692039 14.6190873,14.6213203 C16.7669709,12.4734367 16.7669709,8.99102935 14.6190873,6.84314575 C12.4712037,4.69526215 8.98879631,4.69526215 6.8409127,6.84314575 Z\" id=\"形状\"></path></g></svg> <!----> <!----> <!----> <!----> <!----></div>\n"
     ]
    }
   ],
   "source": [
    "# 点放大镜搜\n",
    "element = driver.find_element_by_xpath('//button[@class=\"weui-desktop-icon-btn weui-desktop-search__btn\"]')\n",
    "main_content = element.get_attribute('innerHTML')\n",
    "print(main_content)\n",
    "element.click()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<li class=\"inner_link_account_item\"><div class=\"weui-desktop-vm_primary\"><img src=\"http://mmbiz.qpic.cn/mmbiz_png/O4sxropic7DGqrwesWs2wSfkKKIQLSImBW84dxhtcxiciaib73wnYxuCo2VBCQbNZUyE5h15o1Ypw2ia0icZFic3hHAOg/0?wx_fmt=png\" class=\"inner_link_account_avatar\"> <strong class=\"inner_link_account_nickname\">温酒居</strong> <i class=\"inner_link_account_wechat\">微信号：win9wine</i></div> <div class=\"weui-desktop-vm_default inner_link_account_type\">订阅号</div></li><li class=\"inner_link_account_item\"><div class=\"weui-desktop-vm_primary\"><img src=\"http://mmbiz.qpic.cn/mmbiz_png/GGBJJX8Aib5a4wxwIRqssB4jibftOBXzkkGDfKFuQaHSkA2zh61OQwqMiavydQzVKpEbiasjDeMzaUMAwmxEGFCRQw/0?wx_fmt=png\" class=\"inner_link_account_avatar\"> <strong class=\"inner_link_account_nickname\">未名居 温酒师</strong> <i class=\"inner_link_account_wechat\">微信号：suki-Samar</i></div> <div class=\"weui-desktop-vm_default inner_link_account_type\">订阅号</div></li><li class=\"inner_link_account_item\"><div class=\"weui-desktop-vm_primary\"><img src=\"http://mmbiz.qpic.cn/mmbiz_png/llaXFlVFVqg3jSxY1jfvCJuzCaibAnK4ErWArZOlEU1nR5tJmFJA6ASHqftG74ibwlu22FEN7nkwqFKwI87UFgLg/0?wx_fmt=png\" class=\"inner_link_account_avatar\"> <strong class=\"inner_link_account_nickname\">鹿居GRWJ</strong> <i class=\"inner_link_account_wechat\">微信号：GRWJshen0707</i></div> <div class=\"weui-desktop-vm_default inner_link_account_type\">订阅号</div></li><li class=\"inner_link_account_item\"><div class=\"weui-desktop-vm_primary\"><img src=\"http://mmbiz.qpic.cn/mmbiz_png/lB8RlMJ0WPRzXHjVvvNQjbHHxYQicC9KjMHFDWicTSwOYtb8CiaXGv6s6zqTCafYr7f0uYwNhT3wlZoVx8ONXibjxw/0?wx_fmt=png\" class=\"inner_link_account_avatar\"> <strong class=\"inner_link_account_nickname\">画楼西畔温酒居</strong> <i class=\"inner_link_account_wechat\">微信号：hualouxipanwenjiuju</i></div> <div class=\"weui-desktop-vm_default inner_link_account_type\">订阅号</div></li>\n"
     ]
    }
   ],
   "source": [
    "element = driver.find_element_by_xpath('//ul[@class=\"inner_link_account_list\"]')\n",
    "main_content = element.get_attribute('innerHTML')\n",
    "print(main_content)\n",
    "公众号SERP = main_content\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 解析\n",
    "root = fromstring(公众号SERP) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "主 = root.xpath('//li[@class=\"inner_link_account_item\"]')\n",
    "\n",
    "account_list = []\n",
    "for e in 主:\n",
    "    account_nickname = e.xpath('./div/strong[@class=\"inner_link_account_nickname\"]')[0].text\n",
    "    account_wechat = e.xpath('./div/i[@class=\"inner_link_account_wechat\"]')[0].text\n",
    "    account_img = e.xpath('./div/img/@src')[0]\n",
    "    account = {\"nickname\": account_nickname, \"wechat\": account_wechat, \"img\": account_img,}\n",
    "    account_list.append(account)\n",
    "\n",
    "df_account = pd.DataFrame(account_list)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>nickname</th>\n",
       "      <th>wechat</th>\n",
       "      <th>img</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>温酒居</td>\n",
       "      <td>微信号：win9wine</td>\n",
       "      <td>http://mmbiz.qpic.cn/mmbiz_png/O4sxropic7DGqrw...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>未名居 温酒师</td>\n",
       "      <td>微信号：suki-Samar</td>\n",
       "      <td>http://mmbiz.qpic.cn/mmbiz_png/GGBJJX8Aib5a4wx...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>鹿居GRWJ</td>\n",
       "      <td>微信号：GRWJshen0707</td>\n",
       "      <td>http://mmbiz.qpic.cn/mmbiz_png/llaXFlVFVqg3jSx...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>画楼西畔温酒居</td>\n",
       "      <td>微信号：hualouxipanwenjiuju</td>\n",
       "      <td>http://mmbiz.qpic.cn/mmbiz_png/lB8RlMJ0WPRzXHj...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  nickname                   wechat  \\\n",
       "0      温酒居             微信号：win9wine   \n",
       "1  未名居 温酒师           微信号：suki-Samar   \n",
       "2   鹿居GRWJ         微信号：GRWJshen0707   \n",
       "3  画楼西畔温酒居  微信号：hualouxipanwenjiuju   \n",
       "\n",
       "                                                 img  \n",
       "0  http://mmbiz.qpic.cn/mmbiz_png/O4sxropic7DGqrw...  \n",
       "1  http://mmbiz.qpic.cn/mmbiz_png/GGBJJX8Aib5a4wx...  \n",
       "2  http://mmbiz.qpic.cn/mmbiz_png/llaXFlVFVqg3jSx...  \n",
       "3  http://mmbiz.qpic.cn/mmbiz_png/lB8RlMJ0WPRzXHj...  "
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_account"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<div class=\"weui-desktop-vm_primary\"><img src=\"http://mmbiz.qpic.cn/mmbiz_png/O4sxropic7DGqrwesWs2wSfkKKIQLSImBW84dxhtcxiciaib73wnYxuCo2VBCQbNZUyE5h15o1Ypw2ia0icZFic3hHAOg/0?wx_fmt=png\" class=\"inner_link_account_avatar\"> <strong class=\"inner_link_account_nickname\">温酒居</strong> <i class=\"inner_link_account_wechat\">微信号：win9wine</i></div> <div class=\"weui-desktop-vm_default inner_link_account_type\">订阅号</div>\n"
     ]
    }
   ],
   "source": [
    "element = driver.find_element_by_xpath('//ul[@class=\"inner_link_account_list\"]/li')\n",
    "main_content = element.get_attribute('innerHTML')\n",
    "print(main_content)\n",
    "element.click()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'\\n跳转_input = driver.find_element_by_xpath(\\'//span[@class=\"weui-desktop-pagination__form\"]/input\\')\\n跳转_a = driver.find_element_by_xpath(\\'//span[@class=\"weui-desktop-pagination__form\"]/a\\')\\n跳转_input.clear()\\n跳转_input.send_keys(2)\\n跳转_a.click()\\n'"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 跳转testing\n",
    "'''\n",
    "跳转_input = driver.find_element_by_xpath('//span[@class=\"weui-desktop-pagination__form\"]/input')\n",
    "跳转_a = driver.find_element_by_xpath('//span[@class=\"weui-desktop-pagination__form\"]/a')\n",
    "跳转_input.clear()\n",
    "跳转_input.send_keys(2)\n",
    "跳转_a.click()\n",
    "'''"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[1, 80]\n",
      "False\n"
     ]
    }
   ],
   "source": [
    "# 跳转上限\n",
    "l_e = driver.find_elements_by_xpath('//label[@class=\"weui-desktop-pagination__num\"]')\n",
    "l_e_int  = [int(x.text) for x in l_e] \n",
    "print (l_e_int)\n",
    "print (l_e_int[0]==l_e_int[-1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80]\n"
     ]
    }
   ],
   "source": [
    "pages = list(range(l_e_int[0],l_e_int[-1]+1 ))\n",
    "#print(pages[0:2])\n",
    "pages = list(range(1,l_e_int[-1]+1 ))\n",
    "print(pages)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 循环/遍历"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "# global varialbes \n",
    "html_raw = dict()\n",
    "main_content =\"\"\n",
    "element = None"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [],
   "source": [
    "def process_pages (pages):\n",
    "    for p in pages:\n",
    "        print (p,end='\\t')\n",
    "\n",
    "        跳转_input = driver.find_element_by_xpath('//span[@class=\"weui-desktop-pagination__form\"]/input')\n",
    "        跳转_a = driver.find_element_by_xpath('//span[@class=\"weui-desktop-pagination__form\"]/a')\n",
    "        跳转_input.clear()\n",
    "        跳转_input.send_keys(p)\n",
    "        跳转_a.click()\n",
    "\n",
    "        time.sleep(1+10*random())\n",
    "\n",
    "        element = driver.find_element_by_xpath('//*[@id=\"vue_app\"]/div[2]/div[1]/div/div[2]/div[2]/form[1]/div[4]/div/div/div[2]/div')\n",
    "        main_content = element.get_attribute('innerHTML')\n",
    "       # print(main_content)\n",
    "        html_raw[p] = main_content\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1\t2\t3\t4\t5\t6\t7\t8\t9\t10\t11\t12\t13\t14\t15\t16\t17\t18\t19\t20\t21\t22\t23\t24\t25\t26\t27\t28\t29\t30\t31\t32\t33\t34\t35\t36\t37\t38\t39\t40\t41\t42\t43\t44\t45\t46\t47\t48\t49\t50\t51\t52\t53\t54\t55\t56\t57\t58\t59\t60\t61\t62\t63\t64\t65\t66\t67\t68\t69\t70\t71\t72\t73\t74\t75\t76\t77\t78\t79\t80\t"
     ]
    }
   ],
   "source": [
    "process_pages(pages)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>html_snippets</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>&lt;div class=\"weui-desktop-radio-group\"&gt;&lt;label c...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>&lt;div class=\"weui-desktop-radio-group\"&gt;&lt;label c...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>&lt;div class=\"weui-desktop-radio-group\"&gt;&lt;label c...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>&lt;div class=\"weui-desktop-radio-group\"&gt;&lt;label c...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>&lt;div class=\"weui-desktop-radio-group\"&gt;&lt;label c...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>76</th>\n",
       "      <td>&lt;div class=\"weui-desktop-radio-group\"&gt;&lt;label c...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>77</th>\n",
       "      <td>&lt;div class=\"weui-desktop-radio-group\"&gt;&lt;label c...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>78</th>\n",
       "      <td>&lt;div class=\"weui-desktop-radio-group\"&gt;&lt;label c...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>79</th>\n",
       "      <td>&lt;div class=\"weui-desktop-radio-group\"&gt;&lt;label c...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>80</th>\n",
       "      <td>&lt;div class=\"weui-desktop-radio-group\"&gt;&lt;label c...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>80 rows × 1 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                        html_snippets\n",
       "1   <div class=\"weui-desktop-radio-group\"><label c...\n",
       "2   <div class=\"weui-desktop-radio-group\"><label c...\n",
       "3   <div class=\"weui-desktop-radio-group\"><label c...\n",
       "4   <div class=\"weui-desktop-radio-group\"><label c...\n",
       "5   <div class=\"weui-desktop-radio-group\"><label c...\n",
       "..                                                ...\n",
       "76  <div class=\"weui-desktop-radio-group\"><label c...\n",
       "77  <div class=\"weui-desktop-radio-group\"><label c...\n",
       "78  <div class=\"weui-desktop-radio-group\"><label c...\n",
       "79  <div class=\"weui-desktop-radio-group\"><label c...\n",
       "80  <div class=\"weui-desktop-radio-group\"><label c...\n",
       "\n",
       "[80 rows x 1 columns]"
      ]
     },
     "execution_count": 47,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = pd.DataFrame([html_raw]).T\n",
    "df.columns = [\"html_snippets\"]\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Stored 'html_raw' (dict)\n"
     ]
    }
   ],
   "source": [
    "%store html_raw\n",
    "import pickle \n",
    "filehandler = open(\"html_raw\", 'wb') \n",
    "pickle.dump(html_raw, filehandler)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "80\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>html_snippets</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>&lt;div class=\"weui-desktop-radio-group\"&gt;&lt;label c...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>&lt;div class=\"weui-desktop-radio-group\"&gt;&lt;label c...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>&lt;div class=\"weui-desktop-radio-group\"&gt;&lt;label c...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>&lt;div class=\"weui-desktop-radio-group\"&gt;&lt;label c...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>&lt;div class=\"weui-desktop-radio-group\"&gt;&lt;label c...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>76</th>\n",
       "      <td>&lt;div class=\"weui-desktop-radio-group\"&gt;&lt;label c...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>77</th>\n",
       "      <td>&lt;div class=\"weui-desktop-radio-group\"&gt;&lt;label c...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>78</th>\n",
       "      <td>&lt;div class=\"weui-desktop-radio-group\"&gt;&lt;label c...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>79</th>\n",
       "      <td>&lt;div class=\"weui-desktop-radio-group\"&gt;&lt;label c...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>80</th>\n",
       "      <td>&lt;div class=\"weui-desktop-radio-group\"&gt;&lt;label c...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>80 rows × 1 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                        html_snippets\n",
       "1   <div class=\"weui-desktop-radio-group\"><label c...\n",
       "2   <div class=\"weui-desktop-radio-group\"><label c...\n",
       "3   <div class=\"weui-desktop-radio-group\"><label c...\n",
       "4   <div class=\"weui-desktop-radio-group\"><label c...\n",
       "5   <div class=\"weui-desktop-radio-group\"><label c...\n",
       "..                                                ...\n",
       "76  <div class=\"weui-desktop-radio-group\"><label c...\n",
       "77  <div class=\"weui-desktop-radio-group\"><label c...\n",
       "78  <div class=\"weui-desktop-radio-group\"><label c...\n",
       "79  <div class=\"weui-desktop-radio-group\"><label c...\n",
       "80  <div class=\"weui-desktop-radio-group\"><label c...\n",
       "\n",
       "[80 rows x 1 columns]"
      ]
     },
     "execution_count": 54,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_out = df[~df.duplicated()]\n",
    "print (len(df_out))\n",
    "df_out"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[1,\n",
       " 2,\n",
       " 3,\n",
       " 4,\n",
       " 5,\n",
       " 6,\n",
       " 7,\n",
       " 8,\n",
       " 9,\n",
       " 10,\n",
       " 11,\n",
       " 12,\n",
       " 13,\n",
       " 14,\n",
       " 15,\n",
       " 16,\n",
       " 17,\n",
       " 18,\n",
       " 19,\n",
       " 20,\n",
       " 21,\n",
       " 22,\n",
       " 23,\n",
       " 24,\n",
       " 25,\n",
       " 26,\n",
       " 27,\n",
       " 28,\n",
       " 29,\n",
       " 30,\n",
       " 31,\n",
       " 32,\n",
       " 33,\n",
       " 34,\n",
       " 35,\n",
       " 36,\n",
       " 37,\n",
       " 38,\n",
       " 39,\n",
       " 40,\n",
       " 41,\n",
       " 42,\n",
       " 43,\n",
       " 44,\n",
       " 45,\n",
       " 46,\n",
       " 47,\n",
       " 48,\n",
       " 49,\n",
       " 50,\n",
       " 51,\n",
       " 52,\n",
       " 53,\n",
       " 54,\n",
       " 55,\n",
       " 56,\n",
       " 57,\n",
       " 58,\n",
       " 59,\n",
       " 60,\n",
       " 61,\n",
       " 62,\n",
       " 63,\n",
       " 64,\n",
       " 65,\n",
       " 66,\n",
       " 67,\n",
       " 68,\n",
       " 69,\n",
       " 70,\n",
       " 71,\n",
       " 72,\n",
       " 73,\n",
       " 74,\n",
       " 75,\n",
       " 76,\n",
       " 77,\n",
       " 78,\n",
       " 79,\n",
       " 80]"
      ]
     },
     "execution_count": 56,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "try_again = list(df_out.index)\n",
    "print(try_again)\n",
    "try_again = try_again + list (set(pages).difference(set(df.index.values)))\n",
    "try_again"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 暂存档"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [],
   "source": [
    "filename = fn [\"output\"] [\"公众号_htm_snippets\"] \n",
    "df_out.to_csv(filename.format(公众号=公众号), sep=\"\\t\", encoding=\"utf8\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [],
   "source": [
    "def parse_html_snippets(_snippet_):\n",
    "    root = fromstring(_snippet_) \n",
    "    title = [x.text for x in root.xpath('//div[@class=\"inner_link_article_title\"]/span[2]')]\n",
    "    create_time = [x.text for x in root.xpath('//div[@class=\"inner_link_article_date\"]')]\n",
    "    link = [x for x in root.xpath('//a/@href')]\n",
    "    _df_ = pd.DataFrame({\"title\":title, \"create_time\": create_time, \"link\":link})\n",
    "    return(_df_)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "5,5,5,5,5,5,5,5,6,6,5,5,5,5,5,5,5,6,5,5,5,5,6,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,6,5,5,5,5,5,6,5,5,5,6,5,5,5,7,10,10,10,5,6,5,5,7,5,5,6,6,2,"
     ]
    }
   ],
   "source": [
    "    \n",
    "l_df = []\n",
    "for p in pages:\n",
    "    _df_ = parse_html_snippets(df.loc[p,\"html_snippets\"])\n",
    "    print (len(_df_), end=\",\")\n",
    "    l_df.append(_df_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>title</th>\n",
       "      <th>create_time</th>\n",
       "      <th>link</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>睡前故事：老兵</td>\n",
       "      <td>2021-02-21</td>\n",
       "      <td>http://mp.weixin.qq.com/s?__biz=MzI0NjA4MTUxNA...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>睡前故事：精灵</td>\n",
       "      <td>2021-02-07</td>\n",
       "      <td>http://mp.weixin.qq.com/s?__biz=MzI0NjA4MTUxNA...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>睡前故事：巨龙</td>\n",
       "      <td>2021-02-06</td>\n",
       "      <td>http://mp.weixin.qq.com/s?__biz=MzI0NjA4MTUxNA...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>睡前故事：生日</td>\n",
       "      <td>2021-01-09</td>\n",
       "      <td>http://mp.weixin.qq.com/s?__biz=MzI0NjA4MTUxNA...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>睡前故事：师父</td>\n",
       "      <td>2020-11-06</td>\n",
       "      <td>http://mp.weixin.qq.com/s?__biz=MzI0NjA4MTUxNA...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>睡前故事：幽灵</td>\n",
       "      <td>2020-10-30</td>\n",
       "      <td>http://mp.weixin.qq.com/s?__biz=MzI0NjA4MTUxNA...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>睡前故事：将军</td>\n",
       "      <td>2020-10-29</td>\n",
       "      <td>http://mp.weixin.qq.com/s?__biz=MzI0NjA4MTUxNA...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>睡前故事：负责</td>\n",
       "      <td>2019-11-09</td>\n",
       "      <td>http://mp.weixin.qq.com/s?__biz=MzI0NjA4MTUxNA...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>睡前故事：小妖</td>\n",
       "      <td>2019-10-18</td>\n",
       "      <td>http://mp.weixin.qq.com/s?__biz=MzI0NjA4MTUxNA...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>睡前故事：影子</td>\n",
       "      <td>2019-09-26</td>\n",
       "      <td>http://mp.weixin.qq.com/s?__biz=MzI0NjA4MTUxNA...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>睡前故事：社交</td>\n",
       "      <td>2019-05-17</td>\n",
       "      <td>http://mp.weixin.qq.com/s?__biz=MzI0NjA4MTUxNA...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      title create_time                                               link\n",
       "0   睡前故事：老兵  2021-02-21  http://mp.weixin.qq.com/s?__biz=MzI0NjA4MTUxNA...\n",
       "1   睡前故事：精灵  2021-02-07  http://mp.weixin.qq.com/s?__biz=MzI0NjA4MTUxNA...\n",
       "2   睡前故事：巨龙  2021-02-06  http://mp.weixin.qq.com/s?__biz=MzI0NjA4MTUxNA...\n",
       "3   睡前故事：生日  2021-01-09  http://mp.weixin.qq.com/s?__biz=MzI0NjA4MTUxNA...\n",
       "4   睡前故事：师父  2020-11-06  http://mp.weixin.qq.com/s?__biz=MzI0NjA4MTUxNA...\n",
       "5   睡前故事：幽灵  2020-10-30  http://mp.weixin.qq.com/s?__biz=MzI0NjA4MTUxNA...\n",
       "6   睡前故事：将军  2020-10-29  http://mp.weixin.qq.com/s?__biz=MzI0NjA4MTUxNA...\n",
       "7   睡前故事：负责  2019-11-09  http://mp.weixin.qq.com/s?__biz=MzI0NjA4MTUxNA...\n",
       "8   睡前故事：小妖  2019-10-18  http://mp.weixin.qq.com/s?__biz=MzI0NjA4MTUxNA...\n",
       "9   睡前故事：影子  2019-09-26  http://mp.weixin.qq.com/s?__biz=MzI0NjA4MTUxNA...\n",
       "10  睡前故事：社交  2019-05-17  http://mp.weixin.qq.com/s?__biz=MzI0NjA4MTUxNA..."
      ]
     },
     "execution_count": 60,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_url_out = pd.concat(l_df).reset_index(drop=True)\n",
    "df_url_out.loc[0:10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>title</th>\n",
       "      <th>create_time</th>\n",
       "      <th>link</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>421</th>\n",
       "      <td>推几首纯音乐</td>\n",
       "      <td>2015-11-26</td>\n",
       "      <td>http://mp.weixin.qq.com/s?__biz=MzI0NjA4MTUxNA...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>422</th>\n",
       "      <td>推几首老歌</td>\n",
       "      <td>2015-11-25</td>\n",
       "      <td>http://mp.weixin.qq.com/s?__biz=MzI0NjA4MTUxNA...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>423</th>\n",
       "      <td>推几首纯音乐</td>\n",
       "      <td>2015-11-24</td>\n",
       "      <td>http://mp.weixin.qq.com/s?__biz=MzI0NjA4MTUxNA...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>424</th>\n",
       "      <td>蝉</td>\n",
       "      <td>2015-11-23</td>\n",
       "      <td>http://mp.weixin.qq.com/s?__biz=MzI0NjA4MTUxNA...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>425</th>\n",
       "      <td>做个小调查</td>\n",
       "      <td>2015-11-23</td>\n",
       "      <td>http://mp.weixin.qq.com/s?__biz=MzI0NjA4MTUxNA...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      title create_time                                               link\n",
       "421  推几首纯音乐  2015-11-26  http://mp.weixin.qq.com/s?__biz=MzI0NjA4MTUxNA...\n",
       "422   推几首老歌  2015-11-25  http://mp.weixin.qq.com/s?__biz=MzI0NjA4MTUxNA...\n",
       "423  推几首纯音乐  2015-11-24  http://mp.weixin.qq.com/s?__biz=MzI0NjA4MTUxNA...\n",
       "424       蝉  2015-11-23  http://mp.weixin.qq.com/s?__biz=MzI0NjA4MTUxNA...\n",
       "425   做个小调查  2015-11-23  http://mp.weixin.qq.com/s?__biz=MzI0NjA4MTUxNA..."
      ]
     },
     "execution_count": 61,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_url_out.tail(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>title</th>\n",
       "      <th>create_time</th>\n",
       "      <th>link</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>value</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>疯狂的外星人：宁浩仍然是宁浩</td>\n",
       "      <td>2019-02-14</td>\n",
       "      <td>http://mp.weixin.qq.com/s?__biz=MzI0NjA4MTUxNA...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28</th>\n",
       "      <td>一个推荐：狄仁杰之四大天王</td>\n",
       "      <td>2018-07-20</td>\n",
       "      <td>http://mp.weixin.qq.com/s?__biz=MzI0NjA4MTUxNA...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>30</th>\n",
       "      <td>我们的暗号：“阿比安吉”</td>\n",
       "      <td>2018-05-24</td>\n",
       "      <td>http://mp.weixin.qq.com/s?__biz=MzI0NjA4MTUxNA...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>37</th>\n",
       "      <td>无常店·子衿   章二</td>\n",
       "      <td>2018-03-06</td>\n",
       "      <td>http://mp.weixin.qq.com/s?__biz=MzI0NjA4MTUxNA...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>38</th>\n",
       "      <td>恐惧囚笼·朔月  章一：离群索居者</td>\n",
       "      <td>2018-03-01</td>\n",
       "      <td>http://mp.weixin.qq.com/s?__biz=MzI0NjA4MTUxNA...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>39</th>\n",
       "      <td>无常店·子衿   章一：孟婆</td>\n",
       "      <td>2018-02-28</td>\n",
       "      <td>http://mp.weixin.qq.com/s?__biz=MzI0NjA4MTUxNA...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>40</th>\n",
       "      <td>恐惧囚笼·朔月  楔子</td>\n",
       "      <td>2018-02-27</td>\n",
       "      <td>http://mp.weixin.qq.com/s?__biz=MzI0NjA4MTUxNA...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>41</th>\n",
       "      <td>无常店·子衿   楔子</td>\n",
       "      <td>2018-02-26</td>\n",
       "      <td>http://mp.weixin.qq.com/s?__biz=MzI0NjA4MTUxNA...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>42</th>\n",
       "      <td>近期更新说明</td>\n",
       "      <td>2018-02-26</td>\n",
       "      <td>http://mp.weixin.qq.com/s?__biz=MzI0NjA4MTUxNA...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>43</th>\n",
       "      <td>又是一年生日季</td>\n",
       "      <td>2018-01-10</td>\n",
       "      <td>http://mp.weixin.qq.com/s?__biz=MzI0NjA4MTUxNA...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>46</th>\n",
       "      <td>公主：生活此刻尽狂欢</td>\n",
       "      <td>2017-12-20</td>\n",
       "      <td>http://mp.weixin.qq.com/s?__biz=MzI0NjA4MTUxNA...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>49</th>\n",
       "      <td>端脑</td>\n",
       "      <td>2017-12-09</td>\n",
       "      <td>http://mp.weixin.qq.com/s?__biz=MzI0NjA4MTUxNA...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>77</th>\n",
       "      <td>礼物</td>\n",
       "      <td>2017-06-05</td>\n",
       "      <td>http://mp.weixin.qq.com/s?__biz=MzI0NjA4MTUxNA...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>88</th>\n",
       "      <td>寻找未知</td>\n",
       "      <td>2017-05-06</td>\n",
       "      <td>http://mp.weixin.qq.com/s?__biz=MzI0NjA4MTUxNA...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>113</th>\n",
       "      <td>去你心里</td>\n",
       "      <td>2017-02-20</td>\n",
       "      <td>http://mp.weixin.qq.com/s?__biz=MzI0NjA4MTUxNA...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>114</th>\n",
       "      <td>键盘侠</td>\n",
       "      <td>2017-02-20</td>\n",
       "      <td>http://mp.weixin.qq.com/s?__biz=MzI0NjA4MTUxNA...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>115</th>\n",
       "      <td>王爷与大少爷</td>\n",
       "      <td>2017-02-19</td>\n",
       "      <td>http://mp.weixin.qq.com/s?__biz=MzI0NjA4MTUxNA...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>116</th>\n",
       "      <td>巨龙与公主</td>\n",
       "      <td>2017-02-13</td>\n",
       "      <td>http://mp.weixin.qq.com/s?__biz=MzI0NjA4MTUxNA...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>117</th>\n",
       "      <td>又老了一岁</td>\n",
       "      <td>2017-01-11</td>\n",
       "      <td>http://mp.weixin.qq.com/s?__biz=MzI0NjA4MTUxNA...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>192</th>\n",
       "      <td>一个道歉...</td>\n",
       "      <td>2016-08-20</td>\n",
       "      <td>http://mp.weixin.qq.com/s?__biz=MzI0NjA4MTUxNA...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>250</th>\n",
       "      <td>昨天的标题图</td>\n",
       "      <td>2016-06-06</td>\n",
       "      <td>http://mp.weixin.qq.com/s?__biz=MzI0NjA4MTUxNA...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>267</th>\n",
       "      <td>无常店·食神</td>\n",
       "      <td>2016-05-20</td>\n",
       "      <td>http://mp.weixin.qq.com/s?__biz=MzI0NjA4MTUxNA...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>295</th>\n",
       "      <td>无常店·鲸落</td>\n",
       "      <td>2016-04-23</td>\n",
       "      <td>http://mp.weixin.qq.com/s?__biz=MzI0NjA4MTUxNA...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>318</th>\n",
       "      <td>无常店·算命师</td>\n",
       "      <td>2016-04-01</td>\n",
       "      <td>http://mp.weixin.qq.com/s?__biz=MzI0NjA4MTUxNA...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>344</th>\n",
       "      <td>AI是否即将威胁人类生存？</td>\n",
       "      <td>2016-03-09</td>\n",
       "      <td>http://mp.weixin.qq.com/s?__biz=MzI0NjA4MTUxNA...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>354</th>\n",
       "      <td>无常</td>\n",
       "      <td>2016-03-04</td>\n",
       "      <td>http://mp.weixin.qq.com/s?__biz=MzI0NjA4MTUxNA...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>383</th>\n",
       "      <td>去年最受欢迎的那些句子</td>\n",
       "      <td>2016-02-14</td>\n",
       "      <td>http://mp.weixin.qq.com/s?__biz=MzI0NjA4MTUxNA...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>384</th>\n",
       "      <td>福利</td>\n",
       "      <td>2016-02-11</td>\n",
       "      <td>http://mp.weixin.qq.com/s?__biz=MzI0NjA4MTUxNA...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>385</th>\n",
       "      <td>大圣的徒弟</td>\n",
       "      <td>2016-02-10</td>\n",
       "      <td>http://mp.weixin.qq.com/s?__biz=MzI0NjA4MTUxNA...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>386</th>\n",
       "      <td>拯救者</td>\n",
       "      <td>2016-01-24</td>\n",
       "      <td>http://mp.weixin.qq.com/s?__biz=MzI0NjA4MTUxNA...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>388</th>\n",
       "      <td>众神屠戮者：章七</td>\n",
       "      <td>2016-01-19</td>\n",
       "      <td>http://mp.weixin.qq.com/s?__biz=MzI0NjA4MTUxNA...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>389</th>\n",
       "      <td>兽人永不为奴 Part.6</td>\n",
       "      <td>2016-01-18</td>\n",
       "      <td>http://mp.weixin.qq.com/s?__biz=MzI0NjA4MTUxNA...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>390</th>\n",
       "      <td>兽人永不为奴 Part.5</td>\n",
       "      <td>2016-01-17</td>\n",
       "      <td>http://mp.weixin.qq.com/s?__biz=MzI0NjA4MTUxNA...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>391</th>\n",
       "      <td>兽人永不为奴 Part.4</td>\n",
       "      <td>2016-01-07</td>\n",
       "      <td>http://mp.weixin.qq.com/s?__biz=MzI0NjA4MTUxNA...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>392</th>\n",
       "      <td>兽人永不为奴 Part.3</td>\n",
       "      <td>2016-01-06</td>\n",
       "      <td>http://mp.weixin.qq.com/s?__biz=MzI0NjA4MTUxNA...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>393</th>\n",
       "      <td>众神屠戮者：章六</td>\n",
       "      <td>2016-01-05</td>\n",
       "      <td>http://mp.weixin.qq.com/s?__biz=MzI0NjA4MTUxNA...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>394</th>\n",
       "      <td>兽人永不为奴 Part.2</td>\n",
       "      <td>2016-01-03</td>\n",
       "      <td>http://mp.weixin.qq.com/s?__biz=MzI0NjA4MTUxNA...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>395</th>\n",
       "      <td>众神屠戮者：章六</td>\n",
       "      <td>2016-01-02</td>\n",
       "      <td>http://mp.weixin.qq.com/s?__biz=MzI0NjA4MTUxNA...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>396</th>\n",
       "      <td>众神屠戮者：章五</td>\n",
       "      <td>2015-12-31</td>\n",
       "      <td>http://mp.weixin.qq.com/s?__biz=MzI0NjA4MTUxNA...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>397</th>\n",
       "      <td>兽人永不为奴</td>\n",
       "      <td>2015-12-31</td>\n",
       "      <td>http://mp.weixin.qq.com/s?__biz=MzI0NjA4MTUxNA...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>398</th>\n",
       "      <td>酒窝，欢迎来玩</td>\n",
       "      <td>2015-12-31</td>\n",
       "      <td>http://mp.weixin.qq.com/s?__biz=MzI0NjA4MTUxNA...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>399</th>\n",
       "      <td>诸神屠戮者：章四</td>\n",
       "      <td>2015-12-29</td>\n",
       "      <td>http://mp.weixin.qq.com/s?__biz=MzI0NjA4MTUxNA...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>400</th>\n",
       "      <td>为何希腊神称号最多？</td>\n",
       "      <td>2015-12-27</td>\n",
       "      <td>http://mp.weixin.qq.com/s?__biz=MzI0NjA4MTUxNA...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>401</th>\n",
       "      <td>众神屠戮者 06</td>\n",
       "      <td>2015-12-26</td>\n",
       "      <td>http://mp.weixin.qq.com/s?__biz=MzI0NjA4MTUxNA...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>402</th>\n",
       "      <td>诸神屠戮者 05</td>\n",
       "      <td>2015-12-25</td>\n",
       "      <td>http://mp.weixin.qq.com/s?__biz=MzI0NjA4MTUxNA...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>403</th>\n",
       "      <td>众神屠戮者 04</td>\n",
       "      <td>2015-12-24</td>\n",
       "      <td>http://mp.weixin.qq.com/s?__biz=MzI0NjA4MTUxNA...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>404</th>\n",
       "      <td>众神屠戮者 03</td>\n",
       "      <td>2015-12-23</td>\n",
       "      <td>http://mp.weixin.qq.com/s?__biz=MzI0NjA4MTUxNA...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>405</th>\n",
       "      <td>众神屠戮者 02</td>\n",
       "      <td>2015-12-22</td>\n",
       "      <td>http://mp.weixin.qq.com/s?__biz=MzI0NjA4MTUxNA...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>406</th>\n",
       "      <td>众神屠戮者 01</td>\n",
       "      <td>2015-12-21</td>\n",
       "      <td>http://mp.weixin.qq.com/s?__biz=MzI0NjA4MTUxNA...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>407</th>\n",
       "      <td>安利给大家一个视频</td>\n",
       "      <td>2015-12-19</td>\n",
       "      <td>http://mp.weixin.qq.com/s?__biz=MzI0NjA4MTUxNA...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>409</th>\n",
       "      <td>推几首民谣</td>\n",
       "      <td>2015-12-16</td>\n",
       "      <td>http://mp.weixin.qq.com/s?__biz=MzI0NjA4MTUxNA...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>410</th>\n",
       "      <td>有关中二病</td>\n",
       "      <td>2015-12-08</td>\n",
       "      <td>http://mp.weixin.qq.com/s?__biz=MzI0NjA4MTUxNA...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>411</th>\n",
       "      <td>侠士隐锋，莽夫露刃</td>\n",
       "      <td>2015-12-07</td>\n",
       "      <td>http://mp.weixin.qq.com/s?__biz=MzI0NjA4MTUxNA...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>412</th>\n",
       "      <td>父爱如山</td>\n",
       "      <td>2015-12-05</td>\n",
       "      <td>http://mp.weixin.qq.com/s?__biz=MzI0NjA4MTUxNA...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>413</th>\n",
       "      <td>现代史中的世态炎凉</td>\n",
       "      <td>2015-12-04</td>\n",
       "      <td>http://mp.weixin.qq.com/s?__biz=MzI0NjA4MTUxNA...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>415</th>\n",
       "      <td>真视之眼</td>\n",
       "      <td>2015-11-30</td>\n",
       "      <td>http://mp.weixin.qq.com/s?__biz=MzI0NjA4MTUxNA...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>418</th>\n",
       "      <td>所谓相貌与才华</td>\n",
       "      <td>2015-11-28</td>\n",
       "      <td>http://mp.weixin.qq.com/s?__biz=MzI0NjA4MTUxNA...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>424</th>\n",
       "      <td>蝉</td>\n",
       "      <td>2015-11-23</td>\n",
       "      <td>http://mp.weixin.qq.com/s?__biz=MzI0NjA4MTUxNA...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                   title create_time  \\\n",
       "value                                  \n",
       "14        疯狂的外星人：宁浩仍然是宁浩  2019-02-14   \n",
       "28         一个推荐：狄仁杰之四大天王  2018-07-20   \n",
       "30          我们的暗号：“阿比安吉”  2018-05-24   \n",
       "37           无常店·子衿   章二  2018-03-06   \n",
       "38     恐惧囚笼·朔月  章一：离群索居者  2018-03-01   \n",
       "39        无常店·子衿   章一：孟婆  2018-02-28   \n",
       "40           恐惧囚笼·朔月  楔子  2018-02-27   \n",
       "41           无常店·子衿   楔子  2018-02-26   \n",
       "42                近期更新说明  2018-02-26   \n",
       "43               又是一年生日季  2018-01-10   \n",
       "46            公主：生活此刻尽狂欢  2017-12-20   \n",
       "49                    端脑  2017-12-09   \n",
       "77                    礼物  2017-06-05   \n",
       "88                  寻找未知  2017-05-06   \n",
       "113                 去你心里  2017-02-20   \n",
       "114                  键盘侠  2017-02-20   \n",
       "115               王爷与大少爷  2017-02-19   \n",
       "116                巨龙与公主  2017-02-13   \n",
       "117                又老了一岁  2017-01-11   \n",
       "192              一个道歉...  2016-08-20   \n",
       "250               昨天的标题图  2016-06-06   \n",
       "267               无常店·食神  2016-05-20   \n",
       "295               无常店·鲸落  2016-04-23   \n",
       "318              无常店·算命师  2016-04-01   \n",
       "344        AI是否即将威胁人类生存？  2016-03-09   \n",
       "354                   无常  2016-03-04   \n",
       "383          去年最受欢迎的那些句子  2016-02-14   \n",
       "384                   福利  2016-02-11   \n",
       "385                大圣的徒弟  2016-02-10   \n",
       "386                  拯救者  2016-01-24   \n",
       "388             众神屠戮者：章七  2016-01-19   \n",
       "389        兽人永不为奴 Part.6  2016-01-18   \n",
       "390        兽人永不为奴 Part.5  2016-01-17   \n",
       "391        兽人永不为奴 Part.4  2016-01-07   \n",
       "392        兽人永不为奴 Part.3  2016-01-06   \n",
       "393             众神屠戮者：章六  2016-01-05   \n",
       "394        兽人永不为奴 Part.2  2016-01-03   \n",
       "395             众神屠戮者：章六  2016-01-02   \n",
       "396             众神屠戮者：章五  2015-12-31   \n",
       "397               兽人永不为奴  2015-12-31   \n",
       "398              酒窝，欢迎来玩  2015-12-31   \n",
       "399             诸神屠戮者：章四  2015-12-29   \n",
       "400           为何希腊神称号最多？  2015-12-27   \n",
       "401             众神屠戮者 06  2015-12-26   \n",
       "402             诸神屠戮者 05  2015-12-25   \n",
       "403             众神屠戮者 04  2015-12-24   \n",
       "404             众神屠戮者 03  2015-12-23   \n",
       "405             众神屠戮者 02  2015-12-22   \n",
       "406             众神屠戮者 01  2015-12-21   \n",
       "407            安利给大家一个视频  2015-12-19   \n",
       "409                推几首民谣  2015-12-16   \n",
       "410                有关中二病  2015-12-08   \n",
       "411            侠士隐锋，莽夫露刃  2015-12-07   \n",
       "412                 父爱如山  2015-12-05   \n",
       "413            现代史中的世态炎凉  2015-12-04   \n",
       "415                 真视之眼  2015-11-30   \n",
       "418              所谓相貌与才华  2015-11-28   \n",
       "424                    蝉  2015-11-23   \n",
       "\n",
       "                                                    link  \n",
       "value                                                     \n",
       "14     http://mp.weixin.qq.com/s?__biz=MzI0NjA4MTUxNA...  \n",
       "28     http://mp.weixin.qq.com/s?__biz=MzI0NjA4MTUxNA...  \n",
       "30     http://mp.weixin.qq.com/s?__biz=MzI0NjA4MTUxNA...  \n",
       "37     http://mp.weixin.qq.com/s?__biz=MzI0NjA4MTUxNA...  \n",
       "38     http://mp.weixin.qq.com/s?__biz=MzI0NjA4MTUxNA...  \n",
       "39     http://mp.weixin.qq.com/s?__biz=MzI0NjA4MTUxNA...  \n",
       "40     http://mp.weixin.qq.com/s?__biz=MzI0NjA4MTUxNA...  \n",
       "41     http://mp.weixin.qq.com/s?__biz=MzI0NjA4MTUxNA...  \n",
       "42     http://mp.weixin.qq.com/s?__biz=MzI0NjA4MTUxNA...  \n",
       "43     http://mp.weixin.qq.com/s?__biz=MzI0NjA4MTUxNA...  \n",
       "46     http://mp.weixin.qq.com/s?__biz=MzI0NjA4MTUxNA...  \n",
       "49     http://mp.weixin.qq.com/s?__biz=MzI0NjA4MTUxNA...  \n",
       "77     http://mp.weixin.qq.com/s?__biz=MzI0NjA4MTUxNA...  \n",
       "88     http://mp.weixin.qq.com/s?__biz=MzI0NjA4MTUxNA...  \n",
       "113    http://mp.weixin.qq.com/s?__biz=MzI0NjA4MTUxNA...  \n",
       "114    http://mp.weixin.qq.com/s?__biz=MzI0NjA4MTUxNA...  \n",
       "115    http://mp.weixin.qq.com/s?__biz=MzI0NjA4MTUxNA...  \n",
       "116    http://mp.weixin.qq.com/s?__biz=MzI0NjA4MTUxNA...  \n",
       "117    http://mp.weixin.qq.com/s?__biz=MzI0NjA4MTUxNA...  \n",
       "192    http://mp.weixin.qq.com/s?__biz=MzI0NjA4MTUxNA...  \n",
       "250    http://mp.weixin.qq.com/s?__biz=MzI0NjA4MTUxNA...  \n",
       "267    http://mp.weixin.qq.com/s?__biz=MzI0NjA4MTUxNA...  \n",
       "295    http://mp.weixin.qq.com/s?__biz=MzI0NjA4MTUxNA...  \n",
       "318    http://mp.weixin.qq.com/s?__biz=MzI0NjA4MTUxNA...  \n",
       "344    http://mp.weixin.qq.com/s?__biz=MzI0NjA4MTUxNA...  \n",
       "354    http://mp.weixin.qq.com/s?__biz=MzI0NjA4MTUxNA...  \n",
       "383    http://mp.weixin.qq.com/s?__biz=MzI0NjA4MTUxNA...  \n",
       "384    http://mp.weixin.qq.com/s?__biz=MzI0NjA4MTUxNA...  \n",
       "385    http://mp.weixin.qq.com/s?__biz=MzI0NjA4MTUxNA...  \n",
       "386    http://mp.weixin.qq.com/s?__biz=MzI0NjA4MTUxNA...  \n",
       "388    http://mp.weixin.qq.com/s?__biz=MzI0NjA4MTUxNA...  \n",
       "389    http://mp.weixin.qq.com/s?__biz=MzI0NjA4MTUxNA...  \n",
       "390    http://mp.weixin.qq.com/s?__biz=MzI0NjA4MTUxNA...  \n",
       "391    http://mp.weixin.qq.com/s?__biz=MzI0NjA4MTUxNA...  \n",
       "392    http://mp.weixin.qq.com/s?__biz=MzI0NjA4MTUxNA...  \n",
       "393    http://mp.weixin.qq.com/s?__biz=MzI0NjA4MTUxNA...  \n",
       "394    http://mp.weixin.qq.com/s?__biz=MzI0NjA4MTUxNA...  \n",
       "395    http://mp.weixin.qq.com/s?__biz=MzI0NjA4MTUxNA...  \n",
       "396    http://mp.weixin.qq.com/s?__biz=MzI0NjA4MTUxNA...  \n",
       "397    http://mp.weixin.qq.com/s?__biz=MzI0NjA4MTUxNA...  \n",
       "398    http://mp.weixin.qq.com/s?__biz=MzI0NjA4MTUxNA...  \n",
       "399    http://mp.weixin.qq.com/s?__biz=MzI0NjA4MTUxNA...  \n",
       "400    http://mp.weixin.qq.com/s?__biz=MzI0NjA4MTUxNA...  \n",
       "401    http://mp.weixin.qq.com/s?__biz=MzI0NjA4MTUxNA...  \n",
       "402    http://mp.weixin.qq.com/s?__biz=MzI0NjA4MTUxNA...  \n",
       "403    http://mp.weixin.qq.com/s?__biz=MzI0NjA4MTUxNA...  \n",
       "404    http://mp.weixin.qq.com/s?__biz=MzI0NjA4MTUxNA...  \n",
       "405    http://mp.weixin.qq.com/s?__biz=MzI0NjA4MTUxNA...  \n",
       "406    http://mp.weixin.qq.com/s?__biz=MzI0NjA4MTUxNA...  \n",
       "407    http://mp.weixin.qq.com/s?__biz=MzI0NjA4MTUxNA...  \n",
       "409    http://mp.weixin.qq.com/s?__biz=MzI0NjA4MTUxNA...  \n",
       "410    http://mp.weixin.qq.com/s?__biz=MzI0NjA4MTUxNA...  \n",
       "411    http://mp.weixin.qq.com/s?__biz=MzI0NjA4MTUxNA...  \n",
       "412    http://mp.weixin.qq.com/s?__biz=MzI0NjA4MTUxNA...  \n",
       "413    http://mp.weixin.qq.com/s?__biz=MzI0NjA4MTUxNA...  \n",
       "415    http://mp.weixin.qq.com/s?__biz=MzI0NjA4MTUxNA...  \n",
       "418    http://mp.weixin.qq.com/s?__biz=MzI0NjA4MTUxNA...  \n",
       "424    http://mp.weixin.qq.com/s?__biz=MzI0NjA4MTUxNA...  "
      ]
     },
     "execution_count": 76,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# tagging 标记\n",
    "tagging_list = [\"\",\"妖怪\",\"科幻\",\"冠名\",\"故事\",\"音乐\",\"歌\",\"调查\"] #overwritable\n",
    "\n",
    "v_v_list = []\n",
    "\n",
    "for tag in tagging_list:\n",
    "    index_list = df_url_out [ df_url_out.title.str.contains(tag) ].index.to_list()\n",
    "    v_v_pairs = pd.DataFrame({tag:index_list}).melt().set_index(\"value\")\n",
    "    v_v_list.append(v_v_pairs)\n",
    "\n",
    "df_cat = v_v_list[0]\n",
    "for d in v_v_list:\n",
    "    df_cat.update(d)\n",
    "    \n",
    "# 尚未标记内容\n",
    "df_url_out.loc [ df_cat.query('variable==\"\"').index ]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'http://mp.weixin.qq.com/s?__biz=MzI0NjA4MTUxNA==&mid=2653022802&idx=1&sn=2f44b147b8b0d47766ed97e04afe1daf&chksm=f291f8d7c5e671c1d6f9070171e7d0804dd6c521c0c6d7f5e4851f15909792b6e262b61ea016#rd'"
      ]
     },
     "execution_count": 77,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_url_out.loc[53].link"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>title</th>\n",
       "      <th>create_time</th>\n",
       "      <th>link</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>睡前故事：老兵</td>\n",
       "      <td>2021-02-21</td>\n",
       "      <td>http://mp.weixin.qq.com/s?__biz=MzI0NjA4MTUxNA...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>睡前故事：精灵</td>\n",
       "      <td>2021-02-07</td>\n",
       "      <td>http://mp.weixin.qq.com/s?__biz=MzI0NjA4MTUxNA...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>睡前故事：巨龙</td>\n",
       "      <td>2021-02-06</td>\n",
       "      <td>http://mp.weixin.qq.com/s?__biz=MzI0NjA4MTUxNA...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>睡前故事：生日</td>\n",
       "      <td>2021-01-09</td>\n",
       "      <td>http://mp.weixin.qq.com/s?__biz=MzI0NjA4MTUxNA...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>睡前故事：师父</td>\n",
       "      <td>2020-11-06</td>\n",
       "      <td>http://mp.weixin.qq.com/s?__biz=MzI0NjA4MTUxNA...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>421</th>\n",
       "      <td>推几首纯音乐</td>\n",
       "      <td>2015-11-26</td>\n",
       "      <td>http://mp.weixin.qq.com/s?__biz=MzI0NjA4MTUxNA...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>422</th>\n",
       "      <td>推几首老歌</td>\n",
       "      <td>2015-11-25</td>\n",
       "      <td>http://mp.weixin.qq.com/s?__biz=MzI0NjA4MTUxNA...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>423</th>\n",
       "      <td>推几首纯音乐</td>\n",
       "      <td>2015-11-24</td>\n",
       "      <td>http://mp.weixin.qq.com/s?__biz=MzI0NjA4MTUxNA...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>424</th>\n",
       "      <td>蝉</td>\n",
       "      <td>2015-11-23</td>\n",
       "      <td>http://mp.weixin.qq.com/s?__biz=MzI0NjA4MTUxNA...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>425</th>\n",
       "      <td>做个小调查</td>\n",
       "      <td>2015-11-23</td>\n",
       "      <td>http://mp.weixin.qq.com/s?__biz=MzI0NjA4MTUxNA...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>426 rows × 3 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       title create_time                                               link\n",
       "0    睡前故事：老兵  2021-02-21  http://mp.weixin.qq.com/s?__biz=MzI0NjA4MTUxNA...\n",
       "1    睡前故事：精灵  2021-02-07  http://mp.weixin.qq.com/s?__biz=MzI0NjA4MTUxNA...\n",
       "2    睡前故事：巨龙  2021-02-06  http://mp.weixin.qq.com/s?__biz=MzI0NjA4MTUxNA...\n",
       "3    睡前故事：生日  2021-01-09  http://mp.weixin.qq.com/s?__biz=MzI0NjA4MTUxNA...\n",
       "4    睡前故事：师父  2020-11-06  http://mp.weixin.qq.com/s?__biz=MzI0NjA4MTUxNA...\n",
       "..       ...         ...                                                ...\n",
       "421   推几首纯音乐  2015-11-26  http://mp.weixin.qq.com/s?__biz=MzI0NjA4MTUxNA...\n",
       "422    推几首老歌  2015-11-25  http://mp.weixin.qq.com/s?__biz=MzI0NjA4MTUxNA...\n",
       "423   推几首纯音乐  2015-11-24  http://mp.weixin.qq.com/s?__biz=MzI0NjA4MTUxNA...\n",
       "424        蝉  2015-11-23  http://mp.weixin.qq.com/s?__biz=MzI0NjA4MTUxNA...\n",
       "425    做个小调查  2015-11-23  http://mp.weixin.qq.com/s?__biz=MzI0NjA4MTUxNA...\n",
       "\n",
       "[426 rows x 3 columns]"
      ]
     },
     "execution_count": 78,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_url_out[~df_url_out.duplicated()]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>title</th>\n",
       "      <th>create_time</th>\n",
       "      <th>link</th>\n",
       "      <th>variable</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>睡前故事：老兵</td>\n",
       "      <td>2021-02-21</td>\n",
       "      <td>http://mp.weixin.qq.com/s?__biz=MzI0NjA4MTUxNA...</td>\n",
       "      <td>故事</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>睡前故事：精灵</td>\n",
       "      <td>2021-02-07</td>\n",
       "      <td>http://mp.weixin.qq.com/s?__biz=MzI0NjA4MTUxNA...</td>\n",
       "      <td>故事</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>睡前故事：巨龙</td>\n",
       "      <td>2021-02-06</td>\n",
       "      <td>http://mp.weixin.qq.com/s?__biz=MzI0NjA4MTUxNA...</td>\n",
       "      <td>故事</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>睡前故事：生日</td>\n",
       "      <td>2021-01-09</td>\n",
       "      <td>http://mp.weixin.qq.com/s?__biz=MzI0NjA4MTUxNA...</td>\n",
       "      <td>故事</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>睡前故事：师父</td>\n",
       "      <td>2020-11-06</td>\n",
       "      <td>http://mp.weixin.qq.com/s?__biz=MzI0NjA4MTUxNA...</td>\n",
       "      <td>故事</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>421</th>\n",
       "      <td>推几首纯音乐</td>\n",
       "      <td>2015-11-26</td>\n",
       "      <td>http://mp.weixin.qq.com/s?__biz=MzI0NjA4MTUxNA...</td>\n",
       "      <td>音乐</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>422</th>\n",
       "      <td>推几首老歌</td>\n",
       "      <td>2015-11-25</td>\n",
       "      <td>http://mp.weixin.qq.com/s?__biz=MzI0NjA4MTUxNA...</td>\n",
       "      <td>歌</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>423</th>\n",
       "      <td>推几首纯音乐</td>\n",
       "      <td>2015-11-24</td>\n",
       "      <td>http://mp.weixin.qq.com/s?__biz=MzI0NjA4MTUxNA...</td>\n",
       "      <td>音乐</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>424</th>\n",
       "      <td>蝉</td>\n",
       "      <td>2015-11-23</td>\n",
       "      <td>http://mp.weixin.qq.com/s?__biz=MzI0NjA4MTUxNA...</td>\n",
       "      <td>无法分类</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>425</th>\n",
       "      <td>做个小调查</td>\n",
       "      <td>2015-11-23</td>\n",
       "      <td>http://mp.weixin.qq.com/s?__biz=MzI0NjA4MTUxNA...</td>\n",
       "      <td>调查</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>426 rows × 4 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       title create_time                                               link  \\\n",
       "0    睡前故事：老兵  2021-02-21  http://mp.weixin.qq.com/s?__biz=MzI0NjA4MTUxNA...   \n",
       "1    睡前故事：精灵  2021-02-07  http://mp.weixin.qq.com/s?__biz=MzI0NjA4MTUxNA...   \n",
       "2    睡前故事：巨龙  2021-02-06  http://mp.weixin.qq.com/s?__biz=MzI0NjA4MTUxNA...   \n",
       "3    睡前故事：生日  2021-01-09  http://mp.weixin.qq.com/s?__biz=MzI0NjA4MTUxNA...   \n",
       "4    睡前故事：师父  2020-11-06  http://mp.weixin.qq.com/s?__biz=MzI0NjA4MTUxNA...   \n",
       "..       ...         ...                                                ...   \n",
       "421   推几首纯音乐  2015-11-26  http://mp.weixin.qq.com/s?__biz=MzI0NjA4MTUxNA...   \n",
       "422    推几首老歌  2015-11-25  http://mp.weixin.qq.com/s?__biz=MzI0NjA4MTUxNA...   \n",
       "423   推几首纯音乐  2015-11-24  http://mp.weixin.qq.com/s?__biz=MzI0NjA4MTUxNA...   \n",
       "424        蝉  2015-11-23  http://mp.weixin.qq.com/s?__biz=MzI0NjA4MTUxNA...   \n",
       "425    做个小调查  2015-11-23  http://mp.weixin.qq.com/s?__biz=MzI0NjA4MTUxNA...   \n",
       "\n",
       "    variable  \n",
       "0         故事  \n",
       "1         故事  \n",
       "2         故事  \n",
       "3         故事  \n",
       "4         故事  \n",
       "..       ...  \n",
       "421       音乐  \n",
       "422        歌  \n",
       "423       音乐  \n",
       "424     无法分类  \n",
       "425       调查  \n",
       "\n",
       "[426 rows x 4 columns]"
      ]
     },
     "execution_count": 79,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_o = df_url_out.join(df_cat).replace(\"\", np.nan).fillna(\"无法分类\")\n",
    "df_o"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>title</th>\n",
       "      <th>create_time</th>\n",
       "      <th>link</th>\n",
       "      <th>variable</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>睡前故事：老兵</td>\n",
       "      <td>2021-02-21</td>\n",
       "      <td>http://mp.weixin.qq.com/s?__biz=MzI0NjA4MTUxNA...</td>\n",
       "      <td>故事</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>睡前故事：精灵</td>\n",
       "      <td>2021-02-07</td>\n",
       "      <td>http://mp.weixin.qq.com/s?__biz=MzI0NjA4MTUxNA...</td>\n",
       "      <td>故事</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>睡前故事：巨龙</td>\n",
       "      <td>2021-02-06</td>\n",
       "      <td>http://mp.weixin.qq.com/s?__biz=MzI0NjA4MTUxNA...</td>\n",
       "      <td>故事</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>睡前故事：生日</td>\n",
       "      <td>2021-01-09</td>\n",
       "      <td>http://mp.weixin.qq.com/s?__biz=MzI0NjA4MTUxNA...</td>\n",
       "      <td>故事</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>睡前故事：师父</td>\n",
       "      <td>2020-11-06</td>\n",
       "      <td>http://mp.weixin.qq.com/s?__biz=MzI0NjA4MTUxNA...</td>\n",
       "      <td>故事</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>421</th>\n",
       "      <td>推几首纯音乐</td>\n",
       "      <td>2015-11-26</td>\n",
       "      <td>http://mp.weixin.qq.com/s?__biz=MzI0NjA4MTUxNA...</td>\n",
       "      <td>音乐</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>422</th>\n",
       "      <td>推几首老歌</td>\n",
       "      <td>2015-11-25</td>\n",
       "      <td>http://mp.weixin.qq.com/s?__biz=MzI0NjA4MTUxNA...</td>\n",
       "      <td>歌</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>423</th>\n",
       "      <td>推几首纯音乐</td>\n",
       "      <td>2015-11-24</td>\n",
       "      <td>http://mp.weixin.qq.com/s?__biz=MzI0NjA4MTUxNA...</td>\n",
       "      <td>音乐</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>424</th>\n",
       "      <td>蝉</td>\n",
       "      <td>2015-11-23</td>\n",
       "      <td>http://mp.weixin.qq.com/s?__biz=MzI0NjA4MTUxNA...</td>\n",
       "      <td>无法分类</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>425</th>\n",
       "      <td>做个小调查</td>\n",
       "      <td>2015-11-23</td>\n",
       "      <td>http://mp.weixin.qq.com/s?__biz=MzI0NjA4MTUxNA...</td>\n",
       "      <td>调查</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>426 rows × 4 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       title create_time                                               link  \\\n",
       "0    睡前故事：老兵  2021-02-21  http://mp.weixin.qq.com/s?__biz=MzI0NjA4MTUxNA...   \n",
       "1    睡前故事：精灵  2021-02-07  http://mp.weixin.qq.com/s?__biz=MzI0NjA4MTUxNA...   \n",
       "2    睡前故事：巨龙  2021-02-06  http://mp.weixin.qq.com/s?__biz=MzI0NjA4MTUxNA...   \n",
       "3    睡前故事：生日  2021-01-09  http://mp.weixin.qq.com/s?__biz=MzI0NjA4MTUxNA...   \n",
       "4    睡前故事：师父  2020-11-06  http://mp.weixin.qq.com/s?__biz=MzI0NjA4MTUxNA...   \n",
       "..       ...         ...                                                ...   \n",
       "421   推几首纯音乐  2015-11-26  http://mp.weixin.qq.com/s?__biz=MzI0NjA4MTUxNA...   \n",
       "422    推几首老歌  2015-11-25  http://mp.weixin.qq.com/s?__biz=MzI0NjA4MTUxNA...   \n",
       "423   推几首纯音乐  2015-11-24  http://mp.weixin.qq.com/s?__biz=MzI0NjA4MTUxNA...   \n",
       "424        蝉  2015-11-23  http://mp.weixin.qq.com/s?__biz=MzI0NjA4MTUxNA...   \n",
       "425    做个小调查  2015-11-23  http://mp.weixin.qq.com/s?__biz=MzI0NjA4MTUxNA...   \n",
       "\n",
       "    variable  \n",
       "0         故事  \n",
       "1         故事  \n",
       "2         故事  \n",
       "3         故事  \n",
       "4         故事  \n",
       "..       ...  \n",
       "421       音乐  \n",
       "422        歌  \n",
       "423       音乐  \n",
       "424     无法分类  \n",
       "425       调查  \n",
       "\n",
       "[426 rows x 4 columns]"
      ]
     },
     "execution_count": 80,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_o[~df_o.title.str.contains(\"腾讯\")]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>title</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>variable</th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>故事</th>\n",
       "      <td>357</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>无法分类</th>\n",
       "      <td>58</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>音乐</th>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>歌</th>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>科幻</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>调查</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "          title\n",
       "variable       \n",
       "故事          357\n",
       "无法分类         58\n",
       "音乐            5\n",
       "歌             4\n",
       "科幻            1\n",
       "调查            1"
      ]
     },
     "execution_count": 81,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_stats = df_o.groupby(by=\"variable\").agg({\"title\":\"count\"}).sort_values(by=\"title\", ascending=False)\n",
    "df_stats"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "metadata": {},
   "outputs": [],
   "source": [
    "link_all = df_o[\"link\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>文本内容</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>['原创','','温酒win9','','','','win9wine','想给你讲一万个...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>['原创','','温酒win9','','','','win9wine','想给你讲一万个...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>['原创','','温酒win9','','','','win9wine','想给你讲一万个...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>['原创','','温酒win9','','','','win9wine','想给你讲一万个...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>['原创','','温酒win9','','','','win9wine','想给你讲一万个...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>421</th>\n",
       "      <td>['','','win9wine','想给你讲一万个故事。','','','','','收录...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>422</th>\n",
       "      <td>['','','win9wine','想给你讲一万个故事。','','','','','收录...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>423</th>\n",
       "      <td>['','','win9wine','想给你讲一万个故事。','','','','','收录...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>424</th>\n",
       "      <td>['温酒','','','win9wine','想给你讲一万个故事。','','','','...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>425</th>\n",
       "      <td>['','','win9wine','想给你讲一万个故事。','','','','','收录...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>426 rows × 1 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                  文本内容\n",
       "0    ['原创','','温酒win9','','','','win9wine','想给你讲一万个...\n",
       "1    ['原创','','温酒win9','','','','win9wine','想给你讲一万个...\n",
       "2    ['原创','','温酒win9','','','','win9wine','想给你讲一万个...\n",
       "3    ['原创','','温酒win9','','','','win9wine','想给你讲一万个...\n",
       "4    ['原创','','温酒win9','','','','win9wine','想给你讲一万个...\n",
       "..                                                 ...\n",
       "421  ['','','win9wine','想给你讲一万个故事。','','','','','收录...\n",
       "422  ['','','win9wine','想给你讲一万个故事。','','','','','收录...\n",
       "423  ['','','win9wine','想给你讲一万个故事。','','','','','收录...\n",
       "424  ['温酒','','','win9wine','想给你讲一万个故事。','','','','...\n",
       "425  ['','','win9wine','想给你讲一万个故事。','','','','','收录...\n",
       "\n",
       "[426 rows x 1 columns]"
      ]
     },
     "execution_count": 88,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import requests_html\n",
    "from requests_html import HTMLSession\n",
    "文本内容 = []\n",
    "for i in link_all:\n",
    "    session = HTMLSession()\n",
    "    c = session.get(i)\n",
    "    文本内容.append(str(c.html.xpath('//span/text()')).replace('\\\\xa0','').replace('\\\\n','').replace(' ',''))\n",
    "\n",
    "详细页文本 = pd.DataFrame\n",
    "({\n",
    "    \"文本内容\":文本内容,\n",
    "})\n",
    "详细页文本"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 95,
   "metadata": {},
   "outputs": [],
   "source": [
    "详细页文本.to_excel(\"详细页文本.xlsx\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 输出"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "#with pd.ExcelWriter('{公众号}公众号链接及文章.xlsx'.format(公众号=公众号),mode='w',engine=\"openpyxl\") as writer:  \n",
    "           # df_url_out.to_excel(writer, sheet_name=公众号)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_account.columns.name = \"rel_accounts\"\n",
    "df_o.columns.name = \"url_cat\"\n",
    "df_stats.columns.name = \"stats\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 90,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'stats'"
      ]
     },
     "execution_count": 90,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "_df_.columns.name"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 92,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Get the xlsxwriter workbook and worksheet objects.  \n",
    "with pd.ExcelWriter(fn[\"output\"][\"公众号_xlsx\"].format(公众号=公众号)) as writer:\n",
    "    workbook  = writer.book\n",
    "\n",
    "    for _df_ in [df_account, df_o, df_stats]:\n",
    "        _df_.to_excel(writer, sheet_name = _df_.columns.name)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
